%load_ext autotime

time: 52.9 µs (started: 2024-04-26 03:53:46 -07:00)

%load_ext watermark
%watermark

Last updated: 2024-04-26T03:53:55.133962-07:00

Python implementation: CPython
Python version       : 3.11.8
IPython version      : 8.20.0

Compiler    : GCC 12.3.0
OS          : Linux
Release     : 4.18.0-477.15.1.el8_8.x86_64
Machine     : x86_64
Processor   : x86_64
CPU cores   : 64
Architecture: 64bit

time: 343 ms (started: 2024-04-26 03:53:54 -07:00)

import polars as pl
import warnings
warnings.filterwarnings('ignore')

import os
from rich import print
import datetime

from sklearnex import patch_sklearn
patch_sklearn()

from sklearn.metrics import silhouette_score
from sklearn.cluster import KMeans
from scipy.spatial.distance import cdist

from tqdm import tqdm
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

import ipywidgets as widgets
from ipywidgets import interact, interact_manual

# import panel as pn
# pn.extension()

Intel(R) Extension for Scikit-learn* enabled (https://github.com/intel/scikit-learn-intelex)

time: 50 s (started: 2024-04-26 03:54:06 -07:00)

%watermark --iversions

polars    : 0.20.17
matplotlib: 3.8.2
pandas    : None
ipywidgets: 8.1.1
numpy     : 1.26.3

time: 99 ms (started: 2024-04-26 03:55:01 -07:00)

# READ IN DATA and DESCRIBE
RATING_FILE_PATH = '../data/ratings.csv'  # ml-32m latest curated dataset
df32m = pl.read_csv(RATING_FILE_PATH)
df32m = df32m.with_columns(
    pl.col('movieId').cast(pl.Utf8).cast(pl.Categorical),
    pl.col('userId').cast(pl.Utf8).cast(pl.Categorical),
    pl.from_epoch(pl.col('timestamp'), time_unit='s')
)
with pl.Config(tbl_rows=20):
    print(f"{df32m.describe(percentiles=[.01, .1, .2, .25, .3, .4, .5, .6, .7, .75, .8, .9, .99])=}")

df32m.describe(percentiles=[.01, .1, .2, .25, .3, .4, .5, .6, .7, .75, .8, .9, .99])=shape: (19, 5)
┌────────────┬──────────┬──────────┬─────────────┬────────────────────────────┐
│ statistic  ┆ userId   ┆ movieId  ┆ rating      ┆ timestamp                  │
│ ---        ┆ ---      ┆ ---      ┆ ---         ┆ ---                        │
│ str        ┆ str      ┆ str      ┆ f64         ┆ str                        │
╞════════════╪══════════╪══════════╪═════════════╪════════════════════════════╡
│ count      ┆ 32000204 ┆ 32000204 ┆ 3.2000204e7 ┆ 32000204                   │
│ null_count ┆ 0        ┆ 0        ┆ 0.0         ┆ 0                          │
│ mean       ┆ null     ┆ null     ┆ 3.540396    ┆ 2010-05-30 17:39:59.573263 │
│ std        ┆ null     ┆ null     ┆ 1.058986    ┆ null                       │
│ min        ┆ null     ┆ null     ┆ 0.5         ┆ 1995-01-09 11:46:44        │
│ 1%         ┆ null     ┆ null     ┆ 0.5         ┆ 1996-06-10 17:26:52        │
│ 10%        ┆ null     ┆ null     ┆ 2.0         ┆ 1999-11-17 10:23:31        │
│ 20%        ┆ null     ┆ null     ┆ 3.0         ┆ 2001-08-10 16:24:23        │
│ 25%        ┆ null     ┆ null     ┆ 3.0         ┆ 2003-04-22 11:53:50        │
│ 30%        ┆ null     ┆ null     ┆ 3.0         ┆ 2004-10-16 05:15:06        │
│ 40%        ┆ null     ┆ null     ┆ 3.5         ┆ 2007-01-26 19:22:44        │
│ 50%        ┆ null     ┆ null     ┆ 3.5         ┆ 2010-04-30 10:03:49        │
│ 60%        ┆ null     ┆ null     ┆ 4.0         ┆ 2015-02-01 22:28:39        │
│ 70%        ┆ null     ┆ null     ┆ 4.0         ┆ 2016-10-13 13:27:36        │
│ 75%        ┆ null     ┆ null     ┆ 4.0         ┆ 2017-08-19 15:59:05        │
│ 80%        ┆ null     ┆ null     ┆ 4.5         ┆ 2018-10-03 07:21:40        │
│ 90%        ┆ null     ┆ null     ┆ 5.0         ┆ 2020-11-05 19:45:35        │
│ 99%        ┆ null     ┆ null     ┆ 5.0         ┆ 2023-06-28 02:27:13        │
│ max        ┆ null     ┆ null     ┆ 5.0         ┆ 2023-10-13 02:29:07        │
└────────────┴──────────┴──────────┴─────────────┴────────────────────────────┘

time: 10.6 s (started: 2024-04-26 03:55:21 -07:00)

@interact

########################### Filtering the Raw Dataset: remove obsolete ratings ###########################
#### Filter 1: Recent Ratings, timestamp > cutoff_data = 2014-01-01
#### Filter 2: Movies with more than cut_movie = 200 ratings
#### Filter 3: Users with more than cut_user = 40 ratings
#### We are filtering step by step to guarantee the result aligns with what we need. While filtering all at once
#### (all filters applied on df32m) will give us a much smaller dataset that is not useful.

def show_recent_ratings(cutoff_date = '2014-01-01',
                        cut_movie = widgets.IntSlider(min=1, max=3000, step=1, value=200),
                        cut_user = widgets.IntSlider(min=1, max=3000, step=1, value=40)):
#   cutoff_date        = '2014-01-01'
#   cut_movie          = 200
#   cut_user           = 40
    df32m              = pl.read_csv(RATING_FILE_PATH)
    cutoff_date        = int(datetime.datetime.strptime(cutoff_date, "%Y-%m-%d").replace(tzinfo=datetime.timezone.utc).timestamp())
    f1_df              = df32m.filter(pl.col('timestamp') >= cutoff_date)
    movie_counts       = f1_df.group_by('movieId').agg(pl.len().alias('count'))
    filtered_movie_ids = movie_counts.filter(pl.col('count') >= cut_movie).select('movieId')
    f2_df              = f1_df.join(filtered_movie_ids, on='movieId', how='inner')
    user_counts        = f2_df.group_by('userId').agg(pl.len().alias('count'))
    filtered_user_ids  = user_counts.filter(pl.col('count') >= cut_user).select('userId')
    df                 = f2_df.join(filtered_user_ids, on='userId', how='inner')
    print(f"Original: \t{df32m.shape[0]:,}")
    print(f"Cutting date: \t{f1_df.shape[0]:,}")
    print(f"Cutting movie: \t{f2_df.shape[0]:,}")
    print(f"Cutting user: \t{df.shape[0]:,}")
    print(f"# of Ratings: \t{df.shape[0]:,}")
    print(f"# of Users: \t{df['userId'].n_unique():,}")
    print(f"# of Movies: \t{df['movieId'].n_unique():,}")
    print(f"Sparsity: \t{1-df.shape[0]/df['userId'].n_unique()/df['movieId'].n_unique():.4n}")
    print(f"User Ratings Count: \t{df['userId'].value_counts().describe()}")
    print(f"Movie Ratings Count: \t{df['movieId'].value_counts().describe()}")
    print(df)

interactive(children=(Text(value='2014-01-01', description='cutoff_date'), IntSlider(value=200, description='c…

time: 1.64 s (started: 2024-04-26 03:55:49 -07:00)

# use the picked cutoff values for df and pt
cutoff_date        = '2014-01-01'
cut_movie          = 200
cut_user           = 40
df32m              = pl.read_csv(RATING_FILE_PATH)
cutoff_date        = int(datetime.datetime.strptime(cutoff_date, "%Y-%m-%d").replace(tzinfo=datetime.timezone.utc).timestamp())
f1_df              = df32m.filter(pl.col('timestamp') >= cutoff_date)
df32m              = df32m.with_columns(pl.from_epoch(pl.col('timestamp'), time_unit='s'))
f1_df              = f1_df.with_columns(pl.from_epoch(pl.col('timestamp'), time_unit='s'))
movie_counts       = f1_df.group_by('movieId').agg(pl.len().alias('count')).sort('count', descending=True)
filtered_movie_ids = movie_counts.filter(pl.col('count') >= cut_movie).select('movieId')
f2_df              = f1_df.join(filtered_movie_ids, on='movieId', how='inner')
user_counts        = f2_df.group_by('userId').agg(pl.len().alias('count')).sort('count', descending=True)
filtered_user_ids  = user_counts.filter(pl.col('count') >= cut_user).select('userId')
df                 = f2_df.join(filtered_user_ids, on='userId', how='inner')

time: 972 ms (started: 2024-04-26 03:58:27 -07:00)

movie_counts = movie_counts.join(pl.read_csv('../data/movie_links_87461_title_genre.csv'), on='movieId', how='inner').sort('count', descending=True)

time: 304 ms (started: 2024-04-26 03:58:47 -07:00)

movie_counts

time: 3.96 ms (started: 2024-04-26 03:59:14 -07:00)

user_counts

time: 1.83 ms (started: 2024-04-26 03:59:41 -07:00)

filtered_user_ids

time: 1.66 ms (started: 2024-04-26 04:00:10 -07:00)

filtered_movie_ids

time: 1.59 ms (started: 2024-04-26 05:54:07 -07:00)

df.shape

(11727759, 4)

time: 1.37 ms (started: 2024-04-26 04:00:47 -07:00)

#df.write_csv('../data/ratings_12m_filtered.csv')

time: 1.17 s (started: 2024-04-25 22:50:02 -07:00)

%matplotlib inline
import plotly.express as px
# create a interactive 3D plot, with the x-axis being the 'timestamp', y-axis being the 'movieId', and z-axis being the 'userId'
# let the size of the points be the 'rating'
# add a plane to cut the plot in half, at timestamp=2014-01-01
# sample 1000 points to make the plot more interactive
plt_df = df32m.sample(1000).sort('timestamp')
plt_df = plt_df.with_columns(
    pl.col('movieId').cast(pl.Int64),
    pl.col('userId').cast(pl.Int64),
)
fig = px.scatter_3d(plt_df, x='timestamp', y='movieId', z='userId', size='rating', color='rating')
fig.update_traces(marker=dict(line=dict(width=4, color='DarkSlateGrey')))
fig.update_layout(scene = dict(
                    xaxis_title='timestamp',
                    yaxis_title='movieId',
                    zaxis_title='userId'),
                  showlegend=False)
fig.show()

time: 20.4 s (started: 2024-04-26 04:01:08 -07:00)

%matplotlib inline
import plotly.express as px
# create a interactive 3D plot, with the x-axis being the 'timestamp', y-axis being the 'movieId', and z-axis being the 'userId'
# let the size of the points be the 'rating'
# add a plane to cut the plot in half, at timestamp=2014-01-01
# sample 1000 points to make the plot more interactive
plt_df = f1_df.sample(1000).sort('timestamp')
plt_df = plt_df.with_columns(
    pl.col('movieId').cast(pl.Int64),
    pl.col('userId').cast(pl.Int64),
)
fig = px.scatter_3d(plt_df, x='timestamp', y='movieId', z='userId', size='rating', color='rating')
fig.update_traces(marker=dict(line=dict(width=4, color='DarkSlateGrey')))
fig.update_layout(scene = dict(
                    xaxis_title='timestamp',
                    yaxis_title='movieId',
                    zaxis_title='userId'),
                  showlegend=False)
fig.show()

time: 96.8 ms (started: 2024-04-26 04:01:47 -07:00)

%matplotlib inline
import plotly.express as px
# create a interactive 3D plot, with the x-axis being the 'timestamp', y-axis being the 'movieId', and z-axis being the 'userId'
# let the size of the points be the 'rating'
# add a plane to cut the plot in half, at timestamp=2014-01-01
# sample 1000 points to make the plot more interactive
plt_df = f2_df.sample(1000).sort('timestamp')
plt_df = plt_df.with_columns(
    pl.col('movieId').cast(pl.Int64),
    pl.col('userId').cast(pl.Int64),
)
fig = px.scatter_3d(plt_df, x='timestamp', y='movieId', z='userId', size='rating', color='rating')
fig.update_traces(marker=dict(line=dict(width=4, color='DarkSlateGrey')))
fig.update_layout(scene = dict(
                    xaxis_title='timestamp',
                    yaxis_title='movieId',
                    zaxis_title='userId'),
                  showlegend=False)
fig.show()

time: 41.2 ms (started: 2024-04-26 04:02:23 -07:00)

%matplotlib inline
import plotly.express as px
# create a interactive 3D plot, with the x-axis being the 'timestamp', y-axis being the 'movieId', and z-axis being the 'userId'
# let the size of the points be the 'rating'
# add a plane to cut the plot in half, at timestamp=2014-01-01
# sample 1000 points to make the plot more interactive
plt_df = df.sample(1000).sort('timestamp')
plt_df = plt_df.with_columns(
    pl.col('movieId').cast(pl.Int64),
    pl.col('userId').cast(pl.Int64),
)
fig = px.scatter_3d(plt_df, x='timestamp', y='movieId', z='userId', size='rating', color='rating')
fig.update_traces(marker=dict(line=dict(width=4, color='DarkSlateGrey')))
fig.update_layout(scene = dict(
                    xaxis_title='timestamp',
                    yaxis_title='movieId',
                    zaxis_title='userId'),
                  showlegend=False)
fig.show()

time: 41.5 ms (started: 2024-04-26 04:02:49 -07:00)

# 2d plot

%matplotlib inline
import plotly.express as px
# create a interactive 2D plot, with the x-axis being the 'timestamp', y-axis being the 'movieId', and z-axis being the 'userId'
# let the size of the points be the 'rating'
# add a plane to cut the plot in half, at timestamp=2014-01-01
# sample 1000 points to make the plot more interactive
plt_df = df.sample(1000).sort('timestamp')
plt_df = plt_df.with_columns(
    pl.col('movieId').cast(pl.Int64),
    pl.col('userId').cast(pl.Int64),
)
fig = px.scatter(plt_df, x='movieId', y='userId', size='rating', color='rating')
fig.update_traces(marker=dict(line=dict(width=4, color='DarkSlateGrey')))
fig.update_layout(scene = dict(
                    xaxis_title='movieId',
                    yaxis_title='userId'),
                  showlegend=False)
fig.show()

time: 1.1 s (started: 2024-04-26 04:03:20 -07:00)

pt = df.pivot(index="userId", columns="movieId", values="rating")
# sort most voted users and movies on top
X = pt.select(pl.col('userId'),pl.col(movie_counts.filter(pl.col("movieId").cast(str).is_in(pt.columns))["movieId"].cast(str)))
X = X.with_columns(pl.col('userId').cast(pl.Utf8).cast(pl.Categorical))
filtered_user_ids = filtered_user_ids.with_columns(pl.col('userId').cast(pl.Utf8).cast(pl.Categorical))
X = filtered_user_ids.join(X, on='userId', how='left')
X

time: 3.1 s (started: 2024-04-26 04:07:08 -07:00)

def calculate_sparsity(df: pl.DataFrame) -> float:
    """
    Calculate the sparsity of a Polars DataFrame.
    Args:
    df (pl.DataFrame): The Polars DataFrame for which to calculate sparsity.
    Returns:
    float: The sparsity of the DataFrame.
    """
    # Total elements is the number of rows multiplied by the number of columns
    total_elements = df.height * df.width
    # Count non-missing, non-zero values
    missing = sum(df.null_count())
    # Sparsity is 1 minus the proportion of non-zero, non-missing entries
    sparsity = missing / total_elements
    return sparsity

# Example usage with a dummy DataFrame:
dummy_data = pl.DataFrame({
    'A': [0, 0, 1, None],
    'B': [1, 2, 0, 0],
    'C': [None, None, None, None]
})

calculate_sparsity(dummy_data)

time: 2.27 ms (started: 2024-04-26 11:45:58 -07:00)

print(calculate_sparsity(X[:10000,:]))
print(calculate_sparsity(X[:10000,:1001]))

shape: (1,)
Series: 'userId' [f64]
[
        0.895131
]

shape: (1,)
Series: 'userId' [f64]
[
        0.646962
]

time: 27.8 ms (started: 2024-04-26 11:47:22 -07:00)

# One Centroid represents the whole market. Each Movie's mean rating is its loading for this whole cluster
loadings = X.mean()
loadings

time: 56.8 ms (started: 2024-04-26 05:18:12 -07:00)

# finetuning for the whole market:
# {"system":"use this movie plot to predict the average market reaction in a scale from 0.5 to 5: {{plot}} ", "answer": "{{loading}}"}

# example:
# {"question": "use this move plot "}

 # now we want to find a middle ground
 # a K-group users, with distinction and representation

time: 210 µs (started: 2024-04-26 05:49:23 -07:00)

df_with_titles = df.join(movie_counts, on='movieId')

time: 67.4 ms (started: 2024-04-26 05:55:27 -07:00)

df_with_titles = df.join(movie_counts, on='movieId')

time: 64.2 ms (started: 2024-04-26 06:03:52 -07:00)

# to pandas
df_with_titles_pd = df_with_titles.to_pandas()
df_with_titles_pd = df_with_titles_pd.groupby('userId').apply(lambda x: ';'.join(['{} {}'.format(a, b) for a, b in zip(x['title'], x['rating'])])).reset_index().rename(columns={0:'pattern'})
df_with_titles_pd

time: 12.1 s (started: 2024-04-26 06:55:50 -07:00)

df_with_titles_pd.pattern.apply(lambda x: len(x)).max()

148335

time: 18.2 ms (started: 2024-04-26 06:57:20 -07:00)

df_56k_user_patterns = df_with_titles_pd.set_index('userId')

time: 32.8 ms (started: 2024-04-26 07:32:04 -07:00)

# use openai embeddings to convert patterns into embeddings

import latentscope as ls
# ls.init('../ls_embeddings',openai_key="sk-proj-GlQIJefzzR8hTixZXDO4T3BlbkFJFFDsaVyztKblDis7N4b3") # tier-1 limit
ls.init('../ls_embeddings',openai_key="sk-proj-ENEqiPGjjZ1JwkfBWibTT3BlbkFJ5jxzolr8Z8EFq1pptjRt") # tier-4 limit
ls.ingest('56k-users', df_56k_user_patterns, 'pattern')

Initialized env with data directory at ../ls_embeddings
Loading environment variables from: /dfs6/pub/mgu3/____ML/ml-32m/ML32M10K/code/.env
DATA DIR ../ls_embeddings
DIRECTORY ../ls_embeddings/56k-users
                                             pattern
0  Alien³ (a.k.a. Alien 3) (1992) 3.0;Gremlins (1...
1  Braveheart (1995) 1.0;Bad Boys (1995) 1.5;Forr...
2  Toy Story (1995) 5.0;Twelve Monkeys (a.k.a. 12...
3  Star Wars: Episode IV - A New Hope (1977) 4.0;...
4  Toy Story (1995) 3.0;Star Wars: Episode IV - A...
                                                 pattern
56313  Dead Man Walking (1995) 4.5;Chungking Express ...
56314  Art of War, The (2000) 1.5;Finding Forrester (...
56315  Toy Story (1995) 4.0;Seven (a.k.a. Se7en) (199...
56316  Get Shorty (1995) 4.0;Twelve Monkeys (a.k.a. 1...
56317  Toy Story (1995) 4.0;Braveheart (1995) 4.5;Tax...
Index(['pattern'], dtype='object')
wrote ../ls_embeddings/56k-users/input.parquet
time: 2.83 s (started: 2024-04-26 07:32:44 -07:00)
wrote ../ls_embeddings/56k-users/input.parquet
time: 2.83 s (started: 2024-04-26 07:32:44 -07:00)

%%capture

ls.serve()

time: 3h 51min 57s (started: 2024-04-26 07:32:56 -07:00)

# load openai embeddings 001.h5 and do UMAP and HDBSCAN to get clusters.
# eyeball whether natural clusters exist

X_fill_mean = X.fill_null(strategy='mean')
X_fill_zero = X.fill_null(strategy='zero')
X_fill_min  = X.fill_null(strategy='min')

time: 241 ms (started: 2024-04-26 04:10:16 -07:00)

from sklearn.decomposition import NMF

A = X_fill_zero[:,1:]

# Initialize NMF and fit it to the matrix A
nmf = NMF(n_components=2, init='random', random_state=0)
W = nmf.fit_transform(A)
H = nmf.components_

# Print the resulting matrices
print(f"{W[:2]=}")
print(f"{H[:2]=}")

W:
 [[0.         0.83260542]
 [0.         2.49781625]
 [0.         3.33042167]
 [0.         4.16302709]
 [1.63417451 0.35754022]
 [1.97409297 0.        ]
 [0.81708726 0.17877011]]

H:
 [[0.         0.44234624 0.         2.49355028 2.49355028]
 [1.19564514 1.21171687 1.19564514 0.         0.        ]]

time: 21.4 ms (started: 2024-04-25 22:34:39 -07:00)

from sklearn.decomposition import NMF
from sklearn.model_selection import GridSearchCV
import numpy as np

# Sample data (for example purposes, in practice your data will be different)
X = X_fill_zero[:, 1:]

# Parameters to search over
param_grid = {
    'n_components': [10, 15, 20, 25, 30, 35, 40, 45, 50],  # number of components
    'init': ['random', 'nndsvd'],  # initialization method
    'l1_ratio': [0, 0.5, 1],  # regularization mixing parameter
}

# Note: NMF does not have a 'score' method usually required by GridSearchCV
# Here we create a custom scorer function. In this case, it could be the NMF's
# reconstruction error itself. For the purpose of GridSearchCV, we need to
# make it a function that accepts the estimator, X, and y (which is ignored).
def nmf_score(estimator, X, y=None):
    # Compute the reconstruction error, the lower, the better
    reconstruction_error = np.linalg.norm(X - estimator.transform(X).dot(estimator.components_))
    return -reconstruction_error  # GridSearchCV maximizes the score so return the negative error

# Create the NMF instance
nmf = NMF()

# Create the GridSearchCV instance
grid_search = GridSearchCV(estimator=nmf, param_grid=param_grid, scoring=nmf_score, cv=3)

# Perform grid search (this may take some time depending on your data size and parameter grid)
grid_search.fit(X)

# Best parameters and score
best_params = grid_search.best_params_
best_score = grid_search.best_score_

print("Best parameters found:", best_params)
print("Best score found:", best_score)

Best parameters found:
{'init': 'random', 'l1_ratio': 0, 'n_components': 10}

Best score found: nan

time: 1h 7min 56s (started: 2024-04-25 23:03:26 -07:00)

from sklearn.decomposition import NMF
from sklearn.model_selection import GridSearchCV
import numpy as np

# Sample data (for example purposes, in practice your data will be different)
A = X_fill_zero[:, 1:]

# Parameters to search over
param_grid = {
    'n_components': [8, 9,10, 11, 12, 13, 14, 15],  # number of components
    'init': ['random', 'nndsvd'],  # initialization method
    'l1_ratio': [0, 0.5, 1],  # regularization mixing parameter
}

# Note: NMF does not have a 'score' method usually required by GridSearchCV
# Here we create a custom scorer function. In this case, it could be the NMF's
# reconstruction error itself. For the purpose of GridSearchCV, we need to
# make it a function that accepts the estimator, X, and y (which is ignored).
def nmf_score(estimator, X, y=None):
    # Compute the reconstruction error, the lower, the better
    reconstruction_error = np.linalg.norm(X - estimator.transform(X).dot(estimator.components_))
    return -reconstruction_error  # GridSearchCV maximizes the score so return the negative error

# Create the NMF instance
nmf = NMF()

# Create the GridSearchCV instance
grid_search = GridSearchCV(estimator=nmf, param_grid=param_grid, scoring=nmf_score, cv=3)

# Perform grid search (this may take some time depending on your data size and parameter grid)
grid_search.fit(A)

# Best parameters and score
best_params = grid_search.best_params_
best_score = grid_search.best_score_

print("Best parameters found:", best_params)
print("Best score found:", best_score)

Best parameters found:
{'init': 'random', 'l1_ratio': 0, 'n_components': 8}

Best score found: nan

time: 33min 45s (started: 2024-04-26 04:13:30 -07:00)

# NMF n=10

from sklearn.decomposition import NMF

A = X_fill_zero[:,1:]

# Initialize NMF and fit it to the matrix A
nmf = NMF(n_components=10, init='random', random_state=0)
U = nmf.fit_transform(A)
M = nmf.components_

# Print the resulting matrices
print(f"{U[:2]=}")
print(f"{M[:2]=}")

U[:2]=array([[2.9517392 , 7.56836747, 0.        , 0.60258417, 0.55196251,
        0.        , 0.02226424, 0.97733986, 0.        , 4.84292076],
       [4.40771523, 9.0685571 , 0.        , 0.53739161, 0.68046949,
        0.        , 0.        , 0.61939399, 0.1306364 , 1.24826888]])

M[:2]=array([[0.        , 0.09362535, 0.        , ..., 0.00819534, 0.        ,
        0.02706899],
       [0.13194742, 0.10950007, 0.15303912, ..., 0.01518143, 0.01983315,
        0.00635778]])

time: 19.2 s (started: 2024-04-26 11:51:40 -07:00)

# use U to cluster users into 10 groups
# jump to 2.3 KMeans

time: 184 µs (started: 2024-04-26 11:54:02 -07:00)

print(np.load('../data/U.npy')[:2])
print(np.load('../data/M.npy')[:2])

[[2.9465047  7.57024189 0.         0.60239043 0.55344382 0.
  0.02233019 0.9683429  0.         4.84344654]
 [4.39981346 9.07085948 0.         0.53721187 0.68230197 0.
  0.         0.61367887 0.13100777 1.24841661]]

[[0.         0.09379923 0.         ... 0.00821147 0.         0.02712022]
 [0.13191784 0.10947779 0.153014   ... 0.01517629 0.01982737 0.00635432]]

time: 51.5 ms (started: 2024-04-26 00:23:36 -07:00)

# write U and M to disk,
np.save('../data/U.npy', U)
np.save('../data/M.npy', M)

time: 77.9 ms (started: 2024-04-26 00:21:33 -07:00)

# grid search to find the best svd hyper-parameters
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from sklearn.pipeline import make_pipeline, make_union
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import HistGradientBoostingClassifier
from sklearn.model_selection import RandomizedSearchCV, GridSearchCV
from sklearn.metrics import f1_score, make_scorer

time: 437 µs (started: 2024-04-25 16:41:57 -07:00)

# example SVD n_c=2:
data = X_fill_zero[:,1:]

# Initialize SVD with desired number of components
# For example, reduce dimensions to 2
svd = TruncatedSVD(n_components=2)

# Fit SVD on the data
svd.fit(data)

# Transform the data
reduced_data = svd.transform(data)

# Explained variance ratio of each component
explained_variance_ratio = svd.explained_variance_ratio_

# Explained variance
explained_variance = svd.explained_variance_

# Total variance explained by the two components
total_variance_explained = explained_variance_ratio.sum()

print("Explained Variance Ratio per Component:", explained_variance_ratio)
print("Explained Variance per Component:", explained_variance)
print("Total Variance Explained by all components:", total_variance_explained)

# Optionally, print reduced data shape and data
print("Shape of the reduced data:", reduced_data.shape)
print("Reduced data sample:", reduced_data[:5])

Explained Variance Ratio per Component: [0.09722155 0.03288487]

Explained Variance per Component: [237.2527247   80.24995381]

Total Variance Explained by all components: 0.13010641926714567

Shape of the reduced data:
(56318, 2)

Reduced data sample: [[133.00376388  31.16510487]
 [121.15507989  56.95709056]
 [ 96.17317574   7.36524487]
 [121.23997846  23.14126167]
 [ 94.88499129   6.43712727]]

time: 2.24 s (started: 2024-04-25 20:45:46 -07:00)

U0, S0, Vt0 = np.linalg.svd(X_fill_zero[:,1:], full_matrices=True)
print(f"{U0.shape=}")
print(f"{Vt0.shape=}")
print(f"{S0.shape=}")
print(f"{S0[:10]=}")

U0.shape=(56318, 56318)

Vt0.shape=(5979, 5979)

S0.shape=(5979,)

S0[:10]=array([6237.31217074, 2152.07968921, 2049.07343725, 1619.19600204,
       1521.81169818, 1424.41457779, 1179.46393407, 1129.03339058,
       1041.86336774,  977.87115844])

time: 5min 4s (started: 2024-04-25 21:43:00 -07:00)

U, S, Vt = np.linalg.svd(X_fill_zero[:,1:], full_matrices=False)
print(f"{U.shape=}")
print(f"{Vt.shape=}")
print(f"{S.shape=}")

U.shape=(56318, 5979)

Vt.shape=(5979, 5979)

S.shape=(5979,)

time: 52.1 s (started: 2024-04-25 21:34:59 -07:00)

print(S[:50]) # 50 most significant components

[6237.31217074 2152.07968921 2049.07343725 1619.19600204 1521.81169818
 1424.41457779 1179.46393407 1129.03339058 1041.86336774  977.87115844
  938.79926732  893.71094347  869.90878612  846.77842883  810.2193332
  781.02267472  764.4849044   746.67560439  738.28771328  712.34653437
  689.70132419  672.08246405  654.06569961  649.68152676  632.05935215
  611.58689271  603.23255205  597.63074211  587.64633119  577.87482512
  566.97381058  559.34838485  544.52478823  543.1516472   535.8183286
  526.95349245  521.08248945  512.63979617  507.96459684  503.53502669
  502.19393197  495.46971891  489.1205354   482.4307243   480.28695211
  474.52375232  469.32982393  464.65911358  463.28967386  459.40606028]

time: 3.26 ms (started: 2024-04-25 21:48:11 -07:00)

# Truncated SVD 8,18,28,...,208:
data = X_fill_zero[:,1:]

# Initialize SVD with desired number of components
# For example, reduce dimensions to 2
for n_components in range(8, 208, 10):
    svd = TruncatedSVD(n_components=n_components)
    # Fit SVD on the data
    svd.fit(data)
    # Transform the data
    reduced_data = svd.transform(data)
    # Explained variance ratio of each component
    explained_variance_ratio = svd.explained_variance_ratio_
    # Explained variance
    explained_variance = svd.explained_variance_
    # Total variance explained by the two components
    total_variance_explained = explained_variance_ratio.sum()
    print('*'*8, f'{n_components=}','*'*8)
    print("Explained Variance Ratio per Component:", explained_variance_ratio)
    print("Explained Variance per Component:", explained_variance)
    print("Total Variance Explained by all components:", total_variance_explained)
    # Optionally, print reduced data shape and data
    print("Shape of the reduced data:", reduced_data.shape)
    print("Reduced data sample:", reduced_data[:2])

******** n_components=8 ********

Explained Variance Ratio per Component: [0.09722155 0.03288487 0.02784989 0.01907246 0.01673718 0.01475283
 0.01010284 0.0092745 ]

Explained Variance per Component: [237.2527247   80.24995498  67.96294291  46.54311131  40.84425068
  36.00178855  24.65426763  22.63284597]

Total Variance Explained by all components: 0.22789612304174614

Shape of the reduced data:
(56318, 8)

Reduced data sample: [[133.00376387  31.16636601 123.42716109  15.45883823 -23.44536389
   -6.45489709  12.82095864  48.62831849]
 [121.15507988  56.95791992  85.95726368  -8.90305672   5.83964536
   -1.95302928  -8.05168437   6.24374895]]

******** n_components=18 ********

Explained Variance Ratio per Component: [0.09722155 0.03288487 0.02784989 0.01907246 0.01673716 0.01475282
 0.01010298 0.00927505 0.0078448  0.00695542 0.00637767 0.00568503
 0.00550605 0.00521344 0.00476878 0.00443415 0.00425041 0.00403835]

Explained Variance per Component: [237.2527247   80.24995645  67.96295032  46.54311305  40.84420683
  36.00175104  24.65460742  22.63419888  19.14390046  16.97351371
  15.5636117   13.8733453   13.43657565  12.72250805  11.63739085
  10.82080127  10.37240324   9.85490876]

Total Variance Explained by all components: 0.28297086576977293

pipe = make_pipeline(
    TruncatedSVD(n_components=10),
    memory="cache_svd_n")
grid = GridSearchCV(
    pipe,
    param_grid={
        "truncatedsvd__n_components":np.random.randint(20, 500, 20),
    },
    scoring='explained_variance',
    cv=5,
    refit=False,
    n_jobs=-1)

print(pipe)

grid.fit(X_fill_zero[:,1:])

from sklearn.decomposition import PCA
plt.rcParams["font.family"] = "DejaVu Serif"
covar_matrix = PCA(n_components = len(X_fill_zero.columns)-1) #start from the number of all features we have
covar_matrix.fit(X_fill_zero[:,1:])
plt.ylabel( 'Eigenvalues')
plt.xlabel('# of Features')
plt.title('PCA Eigenvalues')
plt.ylim(0,max(covar_matrix.explained_variance_))
plt.style.context ('seaborn-whitegrid')
plt.plot(covar_matrix.explained_variance_)
plt.show()

time: 8.57 s (started: 2024-04-25 18:52:20 -07:00)

from sklearn.decomposition import PCA
covar_matrix = PCA(n_components = 50) #components are equal to the number of features we have
covar_matrix.fit(X_fill_zero[:,1:])
plt.ylabel( 'Eigenvalues')
plt.xlabel('# of Features')
plt.title('PCA Eigenvalues')
plt.ylim(0,max(covar_matrix.explained_variance_))
plt.style.context ('seaborn-whitegrid')
plt.axvline(x=8, color='r', linestyle='--')
plt.plot(covar_matrix.explained_variance_)
plt.show()

time: 3.92 s (started: 2024-04-25 18:54:46 -07:00)

from sklearn.decomposition import PCA
covar_matrix = PCA(n_components = 10) #components are equal to the number of features we have
covar_matrix.fit(X_fill_zero[:,1:])
kn = KneeLocator(range(0,10), covar_matrix.explained_variance_, curve='convex', direction='decreasing')
print(f"{kn.knee=}")
plt.ylabel( 'Eigenvalues')
plt.xlabel('# of Features')
plt.title('PCA Eigenvalues')
plt.ylim(0,max(covar_matrix.explained_variance_))
plt.style.context ('seaborn-whitegrid')
plt.axvline(x=kn.knee, color='r', linestyle='--')
plt.plot(covar_matrix.explained_variance_)
plt.show()

kn.knee=2

time: 2.32 s (started: 2024-04-25 19:00:01 -07:00)

from kneed import KneeLocator
kn = KneeLocator(range(0,40), covar_matrix.explained_variance_, curve='convex', direction='decreasing')
print(f"{kn.knee=}")

6

time: 3.24 ms (started: 2024-04-25 18:54:19 -07:00)

from kneed import KneeLocator
kn = KneeLocator(range(0,101), covar_matrix.explained_variance_, curve='convex', direction='decreasing')
print(f"{kn.knee=}")

8

time: 2.47 ms (started: 2024-04-25 18:46:08 -07:00)

k = 60
from yellowbrick.cluster import silhouette_visualizer
visualizer = silhouette_visualizer(KMeans(60, random_state=42), X_fill_zero[:,1:])

time: 2min 1s (started: 2024-04-25 18:57:43 -07:00)

k = 2
from yellowbrick.cluster import silhouette_visualizer
visualizer = silhouette_visualizer(KMeans(2, random_state=42), X_fill_zero[:,1:])

time: 1min 59s (started: 2024-04-25 19:00:32 -07:00)

k = 3
from yellowbrick.cluster import silhouette_visualizer
visualizer = silhouette_visualizer(KMeans(3, random_state=42), X_fill_zero[:,1:])

time: 1min 59s (started: 2024-04-25 19:05:32 -07:00)

k = 4
from yellowbrick.cluster import silhouette_visualizer
visualizer = silhouette_visualizer(KMeans(4, random_state=42), X_fill_zero[:,1:])

time: 1min 56s (started: 2024-04-25 19:07:55 -07:00)

k = 5
from yellowbrick.cluster import silhouette_visualizer
visualizer = silhouette_visualizer(KMeans(5, random_state=42), X_fill_zero[:,1:])

time: 1min 55s (started: 2024-04-25 19:10:25 -07:00)

k = 6
from yellowbrick.cluster import silhouette_visualizer
visualizer = silhouette_visualizer(KMeans(6, random_state=42), X_fill_zero[:,1:])

time: 1min 54s (started: 2024-04-25 19:12:26 -07:00)

k = 7
from yellowbrick.cluster import silhouette_visualizer
visualizer = silhouette_visualizer(KMeans(7, random_state=42), X_fill_zero[:,1:])

time: 1min 55s (started: 2024-04-25 19:14:27 -07:00)

k = 8
from yellowbrick.cluster import silhouette_visualizer
visualizer = silhouette_visualizer(KMeans(8, random_state=42), X_fill_zero[:,1:])

time: 1min 57s (started: 2024-04-25 19:02:37 -07:00)

X_fill_mean = X.fill_null(strategy='mean')
X_fill_zero = X.fill_null(strategy='zero')
X_fill_min  = X.fill_null(strategy='min')

time: 236 ms (started: 2024-04-25 15:33:15 -07:00)

X_fill_mean

time: 38.1 ms (started: 2024-04-25 13:45:04 -07:00)

# it is hard to explain why you impute these values. so leave it for now.

time: 174 µs (started: 2024-04-25 16:33:20 -07:00)

# Mean imputer
from sklearn.impute import SimpleImputer
imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean')
df_mean_imputed = pd.DataFrame(imp_mean.fit_transform(df_with_missing),columns = df_with_missing.columns)

time: 8.19 s (started: 2024-04-24 18:15:24 -07:00)

# Most Frequent imputer
from sklearn.impute import SimpleImputer
imp_mfreq = SimpleImputer(missing_values=np.nan, strategy='most_frequent')
df_mfreq_imputed = pd.DataFrame(imp_mfreq.fit_transform(df_with_missing))

time: 8.41 s (started: 2024-04-24 18:15:32 -07:00)

# KNN imputer
from sklearn.impute import KNNImputer
imputer = KNNImputer(n_neighbors=5)
df_knn_imputed = pd.DataFrame(imputer.fit_transform(df_with_missing),columns = df_with_missing.columns)

# quick and dirty. fill in movie mean + random user factor
import miceforest as mf

kds = mf.ImputationKernel(
    df_with_missing,
    datasets=1,
    save_all_iterations=False,
    random_state=420,
)

kds.mice(1)

df_mice_forest_imputed = kds.complete_data()

dir()

# From SVD,        we have U           as the User Embeddings.
# From Imputation, we have X_fill_mean as the User Embeddings.
# From LLMs,       we have U_embed     as the User Embeddings.

# Then, we can run either KMeans or UMAP+HDBSCAN on these UEs to get User Clusters.

U.shape

(56318, 10)

time: 1.19 ms (started: 2024-04-26 11:54:43 -07:00)

kmeans = KMeans(n_clusters=10, random_state=0)
# Fit the KMeans algorithm to the 'U' matrix
kmeans.fit(U)
centroids = kmeans.cluster_centers_
labels = kmeans.labels_
print(f"{centroids.shape=}")
print(f"{labels[:10]=}")

centroids.shape=(10, 10)

labels[:10]=array([1, 1, 1, 1, 1, 1, 1, 1, 9, 1], dtype=int32)

time: 835 ms (started: 2024-04-26 11:58:51 -07:00)

labels.shape # should be 56318x1. plug it back in X to get avg loading for movies

(56318,)

time: 1.26 ms (started: 2024-04-26 12:02:07 -07:00)

np.array(np.unique(labels, return_counts=True)).T

array([[    0, 13843],
       [    1,   262],
       [    2,  2315],
       [    3,  2158],
       [    4, 21798],
       [    5,   876],
       [    6,  5820],
       [    7,  1286],
       [    8,  7293],
       [    9,   667]])

time: 1.86 ms (started: 2024-04-26 12:04:45 -07:00)

# plug labels back to get movie-cluster loadings
X_L = X.with_columns(
    pl.Series('labels', labels)
).group_by('labels', maintain_order=True).mean().select(
    pl.exclude('userId')
)
X_L

time: 83.2 ms (started: 2024-04-26 12:15:18 -07:00)

X_L.write_csv('../data/X_L_movie_loadings_for_10_groups.csv')

time: 40.4 ms (started: 2024-04-26 12:12:16 -07:00)

M_C_L = X_L.with_columns(
    pl.col('labels').cast(pl.Utf8)
    ).transpose(include_header=True, header_name='movieId', column_names='labels')
M_C_L.write_csv('../data/M_C_L_movie_cluster_loadings_5979x11.csv')
M_C_L

time: 42.7 ms (started: 2024-04-26 12:35:52 -07:00)

########### skip to 3 Prepare FT Data ###################

# U.npy (56_318, 10) to do KMeans clustering to get labels

from yellowbrick.cluster import silhouette_visualizer
visualizer = silhouette_visualizer(KMeans(2, random_state=42), U)

time: 55.6 s (started: 2024-04-26 00:36:01 -07:00)

# elbow viz
from yellowbrick.cluster import KElbowVisualizer
vis = KElbowVisualizer(KMeans(), k=range(5,100,5))
vis.fit(U)
vis.show()

<Axes: title={'center': 'Distortion Score Elbow for KMeans Clustering'}, xlabel='k', ylabel='distortion score'>

time: 2.64 s (started: 2024-04-26 00:39:37 -07:00)

# elbow viz
from yellowbrick.cluster import KElbowVisualizer
vis = KElbowVisualizer(KMeans(), k=range(5,100,5), metric='silhouette')
vis.fit(U)
vis.show()

<Axes: title={'center': 'Silhouette Score Elbow for KMeans Clustering'}, xlabel='k', ylabel='silhouette score'>

time: 7min 23s (started: 2024-04-26 00:43:13 -07:00)

# elbow viz
from yellowbrick.cluster import KElbowVisualizer
vis = KElbowVisualizer(KMeans(), k=range(5,100,5), metric='calinski_harabasz')
vis.fit(U)
vis.show()

<Axes: title={'center': 'Calinski Harabasz Score Elbow for KMeans Clustering'}, xlabel='k', ylabel='calinski harabasz score'>

time: 1.27 s (started: 2024-04-26 00:50:45 -07:00)

# elbow viz
from yellowbrick.cluster import KElbowVisualizer
vis = KElbowVisualizer(KMeans(), k=range(20,40), metric='silhouette')
vis.fit(U)
vis.show()

<Axes: title={'center': 'Silhouette Score Elbow for KMeans Clustering'}, xlabel='k', ylabel='silhouette score'>

time: 7min 53s (started: 2024-04-26 00:51:02 -07:00)

# elbow viz
from yellowbrick.cluster import KElbowVisualizer
vis = KElbowVisualizer(KMeans(), k=range(20,40))
vis.fit(U)
vis.show()

<Axes: title={'center': 'Distortion Score Elbow for KMeans Clustering'}, xlabel='k', ylabel='distortion score'>

time: 2.62 s (started: 2024-04-26 01:11:04 -07:00)

# elbow viz
from yellowbrick.cluster import KElbowVisualizer
vis = KElbowVisualizer(KMeans(), k=range(10,40), metric='silhouette')
vis.fit(U)
vis.show()

<Axes: title={'center': 'Silhouette Score Elbow for KMeans Clustering'}, xlabel='k', ylabel='silhouette score'>

time: 11min 54s (started: 2024-04-26 00:59:03 -07:00)

# user embeddings from NMF
U0

array([[-2.13238908e-02,  1.44819752e-02,  6.02356211e-02, ...,
        -2.10993512e-03,  1.06076370e-04, -6.22310116e-04],
       [-1.94242450e-02,  2.64664853e-02,  4.19494100e-02, ...,
        -1.50263473e-03,  4.69035863e-04,  1.42112823e-03],
       [-1.54190095e-02,  3.42269345e-03,  4.71016766e-02, ...,
        -4.83255914e-04,  6.66800463e-04, -4.94009473e-03],
       ...,
       [-1.77246506e-03, -1.52386116e-03, -4.81911236e-03, ...,
         9.96737901e-01,  1.50495452e-04, -1.64694205e-04],
       [-1.21989953e-03,  1.79720073e-03, -2.77891537e-03, ...,
         1.50093534e-04,  9.91962407e-01, -7.39255793e-05],
       [-8.48773956e-04, -2.09768266e-03, -4.46399927e-04, ...,
        -1.63915431e-04,  2.03453663e-04,  9.58858921e-01]])

time: 2.25 ms (started: 2024-04-26 02:18:46 -07:00)

U0.shape

(56318, 56318)

time: 1.12 ms (started: 2024-04-26 03:03:12 -07:00)

from sklearn.datasets import fetch_openml
from sklearn.decomposition import PCA
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

# Dimension reduction and clustering libraries
import umap
import hdbscan
import sklearn.cluster as cluster
from sklearn.metrics import adjusted_rand_score, adjusted_mutual_info_score

2024-04-26 03:03:55.300914: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-04-26 03:03:57.417630: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2024-04-26 03:04:25.388514: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT

time: 1min 29s (started: 2024-04-26 03:03:24 -07:00)

from sklearn.decomposition import NMF

A = X_fill_zero[:,1:]

# Initialize NMF and fit it to the matrix A
nmf = NMF(n_components=5980, init='random', random_state=0)
U = nmf.fit_transform(A)
M = nmf.components_

# Print the resulting matrices
print(f"{U[:2]=}")
print(f"{M[:2]=}")

U[:2]=array([[2.9465047 , 7.57024189, 0.        , 0.60239043, 0.55344382,
        0.        , 0.02233019, 0.9683429 , 0.        , 4.84344654],
       [4.39981346, 9.07085948, 0.        , 0.53721187, 0.68230197,
        0.        , 0.        , 0.61367887, 0.13100777, 1.24841661]])

M[:2]=array([[0.        , 0.09379923, 0.        , ..., 0.00821147, 0.        ,
        0.02712022],
       [0.13191784, 0.10947779, 0.153014  , ..., 0.01517629, 0.01982737,
        0.00635432]])

time: 20.2 s (started: 2024-04-26 00:14:37 -07:00)

# check # of movies have metadata
movies = filtered_movie_ids.join(movie_counts, on='movieId', how='left')
movies

time: 4.75 ms (started: 2024-04-26 12:24:01 -07:00)

plots = pl.read_csv('../../../movieplot/mpst_full_data.csv')
plots = plots.with_columns(
    imdbId=pl.col('imdb_id').str.replace(r'tt0*','').cast(pl.Int64, strict=False)
)
plots

time: 781 ms (started: 2024-04-26 12:24:10 -07:00)

# 1756 null plots, scrape later on IMDb
movies = movies.join(plots, on='imdbId', how='left')
movies['plot_synopsis'].is_null().sum()

1756

time: 3.48 ms (started: 2024-04-26 12:24:33 -07:00)

FT_D = movies.drop_nulls().with_columns(
    pl.col('movieId').cast(pl.Utf8)
    ).join(
        M_C_L,
        on='movieId'
)
FT_D.select(
    ['movieId', 'title', 'genres', 'imdb_id', 'plot_synopsis', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
)

time: 6.34 ms (started: 2024-04-26 12:44:50 -07:00)

FT_D.write_ndjson('../data/FT_D_raw_4225x15.jsonl')

time: 65.7 ms (started: 2024-04-26 12:59:13 -07:00)

# now how to construct the jsonl file format to best represent what we want to do?
# DSPy package:
# https://github.com/stanfordnlp/dspy/blob/main/dspy/teleprompt/finetune.py

# domain expert on movie industry, human interfaces

movieId	count	imdbId	tmdbId	title	genres
i64	u32	i64	i64	str	str
79132	47695	1375666	27205	"Inception (201…	"Action\|Crime\|D…
2571	47209	133093	603	"Matrix, The (1…	"Action\|Sci-Fi\|…
318	44585	111161	278	"Shawshank Rede…	"Crime\|Drama"
58559	42725	468569	155	"Dark Knight, T…	"Action\|Crime\|D…
2959	41295	137523	550	"Fight Club (19…	"Action\|Crime\|D…
…	…	…	…	…	…
292731	1	26812510	1032473	"The Monroy Aff…	"Drama"
292737	1	14907358	986674	"Shelter in Sol…	"Comedy\|Drama"
292753	1	12388280	948139	"Orca (2023)"	"Drama"
292755	1	64027	182776	"The Angry Bree…	"Drama"
292757	1	28995566	1174725	"Race to the Su…	"Action\|Adventu…

userId	count
i64	u32
175325	4985
22744	4473
17035	3984
15875	3898
43703	3837
…	…
170393	1
173537	1
46401	1
65405	1
123995	1

userId	79132	2571	318	58559	2959	7153	4993	356	109487	5952	296	260	593	99114	1196	60069	68954	858	134130	91529	527	1198	68157	3578	72998	106782	112852	59315	4226	6539	6874	4306	33794	6377	74458	1704	…	6770	8928	64278	66304	98956	99106	116660	118760	138104	141648	158956	192109	193954	205857	281904	522	1081	2946	32792	55946	85790	93819	94867	97194	116668	169252	203619	208807	224983	81	3952	7976	67620	115967	127134	185997	251922
cat	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	…	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64
null	4.132816	4.11884	4.328511	4.133716	4.218658	4.03331	4.028187	4.076231	4.112761	4.01789	4.226428	3.957079	4.082426	4.008424	4.046046	3.978485	3.901311	4.219631	4.006142	3.933931	4.08114	3.934357	4.028735	3.956386	3.512107	3.918774	3.897736	3.749119	4.083559	3.688088	3.802861	3.59861	3.869268	3.739308	3.991113	4.135534	…	3.604061	3.3575	3.85	2.524752	3.645078	2.78	3.533854	3.638191	3.4075	3.194737	2.853535	3.8925	2.780612	3.310945	3.255051	3.382653	3.510417	3.435567	3.870466	2.756219	3.387755	3.28	2.95202	3.87	3.206468	3.992386	3.6675	3.4125	3.253769	3.035	3.407035	3.199482	3.492424	3.477273	3.25	2.721939	2.645729

userId	movieId	rating	timestamp	count	imdbId	tmdbId	title	genres
i64	i64	f64	datetime[μs]	u32	i64	i64	str	str
10	1320	3.0	2015-03-22 15:58:19	3390	103644	8077	"Alien³ (a.k.a.…	"Action\|Horror\|…
10	2003	3.5	2017-03-18 20:26:28	3527	87363	927	"Gremlins (1984…	"Comedy\|Horror"
10	2985	3.0	2014-11-01 19:48:25	4936	93870	5548	"RoboCop (1987)…	"Action\|Crime\|D…
10	60040	2.5	2014-08-07 15:45:16	5593	800080	1724	"Incredible Hul…	"Action\|Sci-Fi"
10	68791	2.0	2016-06-11 14:32:44	3984	438488	534	"Terminator Sal…	"Action\|Adventu…
…	…	…	…	…	…	…	…	…
200947	116797	3.5	2017-05-28 12:02:34	24460	2084970	205596	"The Imitation …	"Drama\|Thriller…
200947	122882	3.5	2017-05-28 12:13:52	21315	1392190	76341	"Mad Max: Fury …	"Action\|Adventu…
200947	134853	5.0	2017-05-28 12:11:46	21483	2096673	150540	"Inside Out (20…	"Adventure\|Anim…
200947	152081	3.0	2017-05-28 12:04:17	14431	2948356	269149	"Zootopia (2016…	"Action\|Adventu…
200947	164909	4.5	2017-05-28 12:16:15	7231	3783958	313369	"La La Land (20…	"Comedy\|Drama\|R…

userId	79132	2571	318	58559	2959	7153	4993	356	109487	5952	296	260	593	99114	1196	60069	68954	858	134130	91529	527	1198	68157	3578	72998	106782	112852	59315	4226	6539	6874	4306	33794	6377	74458	1704	…	6770	8928	64278	66304	98956	99106	116660	118760	138104	141648	158956	192109	193954	205857	281904	522	1081	2946	32792	55946	85790	93819	94867	97194	116668	169252	203619	208807	224983	81	3952	7976	67620	115967	127134	185997	251922
cat	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	…	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64
"175325"	4.0	4.0	4.5	4.5	3.5	4.0	4.0	3.5	4.0	4.0	4.5	4.0	4.0	4.0	4.0	4.0	4.0	4.5	4.5	4.0	4.5	4.0	4.0	3.5	3.5	4.0	4.0	3.5	4.0	4.0	4.0	3.5	4.0	4.0	4.0	4.5	…	3.0	3.5	3.5	2.524752	4.0	2.5	3.5	4.0	3.4075	3.0	2.853535	3.5	3.0	3.310945	3.255051	4.0	3.5	3.5	5.0	3.0	4.0	2.5	2.5	3.87	3.5	4.0	3.5	3.4125	3.253769	1.5	3.0	2.0	3.492424	3.0	2.0	3.0	2.645729
"22744"	4.0	5.0	3.0	5.0	5.0	5.0	5.0	4.0	4.0	5.0	5.0	5.0	5.0	3.0	5.0	4.0	4.0	5.0	4.0	5.0	0.5	5.0	3.0	3.0	3.0	4.0	4.0	4.0	4.0	4.0	5.0	5.0	5.0	5.0	3.0	3.0	…	3.604061	3.3575	3.85	2.5	3.645078	4.0	3.533854	3.638191	2.0	2.0	1.0	3.8925	0.5	3.310945	3.255051	0.5	1.0	2.0	3.870466	0.5	3.387755	3.28	3.0	3.87	2.0	3.992386	3.6675	3.4125	3.253769	2.0	3.407035	3.199482	3.492424	2.0	2.0	3.0	2.645729
"17035"	1.5	1.5	5.0	4.5	4.5	4.0	3.0	3.5	2.0	3.5	5.0	0.5	5.0	4.0	0.5	1.0	4.0	5.0	4.0	0.5	4.5	4.5	4.5	4.5	1.5	4.0	0.5	4.0	4.0	3.688088	4.5	4.0	0.5	4.0	4.0	4.0	…	2.0	1.0	3.85	0.5	3.645078	3.5	3.533854	0.5	3.4075	0.5	2.853535	3.8925	2.780612	2.5	3.255051	2.0	4.5	3.435567	3.870466	0.5	2.5	3.0	3.5	3.87	3.206468	3.992386	3.6675	3.5	3.253769	4.0	3.5	3.199482	3.5	2.5	3.5	2.721939	2.645729
"15875"	3.0	5.0	5.0	4.0	5.0	4.0	4.0	3.5	4.0	4.0	5.0	4.0	3.5	4.5	4.5	3.0	3.5	4.0	5.0	4.0	4.0	4.0	5.0	2.5	3.0	4.0	4.0	4.0	5.0	3.0	4.0	4.0	3.5	3.5	4.0	2.5	…	3.0	3.0	3.85	2.524752	3.645078	2.78	3.533854	3.638191	3.4075	3.194737	2.5	4.0	2.0	3.310945	3.255051	4.0	3.0	3.435567	3.0	2.756219	3.387755	3.0	1.5	3.87	3.5	3.992386	3.6675	3.4125	3.253769	3.0	1.5	3.5	3.492424	3.0	3.25	2.721939	2.645729
"43703"	2.5	5.0	3.5	4.0	3.5	3.0	2.5	2.5	3.5	2.5	3.5	3.5	5.0	3.5	5.0	4.0	3.5	4.0	2.5	3.5	3.5	3.5	4.0	2.5	2.0	4.0	2.5	3.5	3.5	3.0	3.5	3.0	3.5	2.5	3.0	3.5	…	1.0	3.0	3.0	1.5	3.645078	2.78	3.533854	3.638191	3.4075	3.194737	2.853535	1.5	2.780612	1.5	3.255051	2.0	3.510417	3.435567	2.0	2.756219	2.5	2.0	2.95202	3.87	2.5	3.992386	2.0	3.4125	2.0	3.035	3.407035	3.199482	3.492424	2.0	3.25	2.721939	1.5
…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…
"114825"	4.132816	4.11884	4.5	4.0	4.5	4.03331	4.028187	4.076231	4.112761	4.01789	4.226428	3.957079	4.0	4.008424	4.046046	3.978485	5.0	5.0	4.006142	3.933931	4.08114	3.934357	4.028735	3.956386	3.512107	3.918774	3.897736	3.749119	4.083559	3.688088	3.802861	3.59861	3.869268	3.739308	3.991113	4.135534	…	3.604061	3.3575	3.85	2.524752	3.645078	2.78	3.533854	3.638191	3.4075	3.194737	2.853535	3.8925	2.780612	3.310945	3.255051	3.382653	3.510417	3.435567	3.870466	2.756219	3.387755	3.28	2.95202	3.87	3.206468	3.992386	3.6675	3.4125	3.253769	3.035	3.407035	3.199482	3.492424	3.477273	3.25	2.721939	2.645729
"60686"	4.0	4.5	4.328511	4.0	5.0	4.03331	4.028187	4.5	4.112761	4.01789	5.0	2.0	5.0	4.0	4.046046	3.978485	3.901311	4.219631	4.006142	3.5	5.0	3.934357	4.0	4.0	2.5	3.918774	3.897736	3.5	4.083559	3.688088	4.5	3.59861	3.869268	3.739308	4.5	4.5	…	3.604061	3.3575	3.85	2.524752	3.645078	2.78	3.533854	3.638191	3.4075	3.194737	2.853535	3.8925	2.780612	3.310945	3.255051	3.382653	3.510417	3.435567	3.870466	2.756219	3.387755	3.28	2.95202	3.87	3.206468	3.992386	3.6675	3.4125	3.253769	3.035	3.407035	3.199482	3.492424	3.477273	3.25	2.721939	2.645729
"71434"	4.132816	4.11884	4.328511	4.133716	4.218658	4.03331	4.028187	4.076231	4.112761	4.01789	5.0	4.5	5.0	4.008424	4.046046	3.978485	3.901311	5.0	4.006142	3.933931	5.0	3.934357	4.028735	3.956386	3.512107	3.918774	3.897736	3.749119	5.0	3.688088	3.802861	3.59861	3.869268	3.739308	3.991113	4.135534	…	3.604061	3.3575	3.85	2.524752	3.645078	2.78	3.533854	3.638191	3.4075	3.194737	2.853535	3.8925	2.780612	3.310945	3.255051	3.382653	3.510417	3.435567	3.870466	2.756219	3.387755	3.28	2.95202	3.87	3.206468	3.992386	3.6675	3.4125	3.253769	3.035	3.407035	3.199482	3.492424	3.477273	3.25	2.721939	2.645729
"57670"	4.5	4.5	4.328511	4.133716	4.218658	4.03331	4.028187	4.076231	4.5	4.01789	4.226428	4.5	4.082426	4.008424	5.0	5.0	4.5	4.219631	4.5	3.933931	4.08114	4.0	4.028735	3.956386	4.0	3.918774	4.5	4.5	4.083559	3.688088	3.802861	3.59861	3.869268	3.739308	3.991113	4.135534	…	3.604061	3.3575	3.85	2.524752	3.645078	2.78	3.533854	3.638191	3.4075	3.194737	2.853535	3.8925	2.780612	3.310945	3.255051	3.382653	3.510417	3.435567	3.870466	2.756219	3.387755	3.28	2.95202	3.87	3.206468	3.992386	3.6675	3.4125	3.253769	3.035	3.407035	3.199482	3.492424	3.477273	3.25	2.721939	2.645729
"121244"	4.132816	2.5	4.5	4.133716	4.218658	4.03331	4.028187	4.076231	4.112761	4.01789	4.226428	3.957079	4.082426	4.008424	4.046046	3.978485	3.901311	4.219631	4.006142	3.933931	4.08114	3.934357	4.028735	3.956386	3.512107	3.918774	3.897736	3.749119	4.083559	3.688088	3.802861	3.59861	3.869268	3.739308	3.991113	4.135534	…	3.604061	3.3575	3.85	2.524752	3.645078	2.78	3.533854	3.638191	3.4075	3.194737	2.853535	3.8925	2.780612	3.310945	3.255051	3.382653	3.510417	3.435567	3.870466	2.756219	3.387755	2.5	2.95202	3.87	3.206468	3.992386	3.6675	3.4125	3.253769	3.035	3.407035	3.199482	3.492424	3.477273	3.25	2.721939	2.645729

0 Load Packages¶

1 Data and Filter¶

1.1 visualize the filtering process¶

1.1.1 the original df32m data¶

1.1.2 cut to 2014-01-01 and later, f1_df¶

1.1.3 cut to movies with at least 200 votes, f2_df¶

1.1.4 cut to users with at least 40 votes, df¶

1.2 Pivot Table¶

2 Clustering¶

2.0 Over-simplifying the market as a whole¶

2.1 Over-complicating, we have personalization¶

2.2 Somewhere in the middle,¶

2.2.1 Not Filling in Missings¶

2.2.1.1 Non-Negative Matrix Factorization¶

Singular Value Decomposition (SVD)¶

Variations such as Truncated SVD, Random SVD, etc.¶

2.2.1.2 Principle Component Analysis¶

2.2.2 Fill in Missing Values¶

All the advanced imputer methods takes weeks to run.¶

2.3 KMeans Clustering¶

2.4 UMAP + HDBSCAN¶

2.4.1 MF Embeddings¶

3 Prepare Finetuning Data¶

4 Finetune a BERT-based model¶

5 Finetune a GPT-based model¶

6 Finetune a Mixtral of Experts model¶

	userId	pattern
0	10	Alien³ (a.k.a. Alien 3) (1992) 3.0;Gremlins (1...
1	16	Braveheart (1995) 1.0;Bad Boys (1995) 1.5;Forr...
2	20	Toy Story (1995) 5.0;Twelve Monkeys (a.k.a. 12...
3	22	Star Wars: Episode IV - A New Hope (1977) 4.0;...
4	23	Toy Story (1995) 3.0;Star Wars: Episode IV - A...
...	...	...
56313	200933	Dead Man Walking (1995) 4.5;Chungking Express ...
56314	200943	Art of War, The (2000) 1.5;Finding Forrester (...
56315	200944	Toy Story (1995) 4.0;Seven (a.k.a. Se7en) (199...
56316	200945	Get Shorty (1995) 4.0;Twelve Monkeys (a.k.a. 1...
56317	200947	Toy Story (1995) 4.0;Braveheart (1995) 4.5;Tax...

labels	79132	2571	318	58559	2959	7153	4993	356	109487	5952	296	260	593	99114	1196	60069	68954	858	134130	91529	527	1198	68157	3578	72998	106782	112852	59315	4226	6539	6874	4306	33794	6377	74458	1704	…	6770	8928	64278	66304	98956	99106	116660	118760	138104	141648	158956	192109	193954	205857	281904	522	1081	2946	32792	55946	85790	93819	94867	97194	116668	169252	203619	208807	224983	81	3952	7976	67620	115967	127134	185997	251922
i32	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	…	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64
1	4.0	4.40856	4.37247	4.2607	4.28937	4.010246	4.058	4.012048	3.91453	3.97561	4.373518	4.243083	4.331395	3.909091	4.301205	3.940265	3.798206	4.245495	3.835586	3.784585	4.118605	4.363636	3.902834	4.037402	3.487854	3.832558	4.044355	3.918	4.165975	3.753968	4.011719	3.672131	3.982213	3.714583	3.652542	3.913717	…	3.275	3.145161	3.5625	2.535714	2.75	2.709677	2.0	2.636364	3.340909	2.625	3.064516	3.25	2.710526	2.970588	3.416667	3.028571	3.322581	3.22	3.53125	2.470588	3.068182	3.272727	3.048077	4.178571	3.155556	3.5	2.722222	3.388889	3.027778	2.945652	3.518868	2.913043	3.603448	3.297872	3.227273	3.157895	2.543478
9	4.374224	4.48367	4.472696	4.412461	4.30737	4.263072	4.225884	4.189097	4.267537	4.22578	4.159785	4.073785	4.050645	4.116838	4.096257	3.979242	3.816148	4.246795	4.194728	4.216216	4.110738	4.038321	3.97265	4.326264	3.958991	3.976407	4.063796	4.078906	4.116959	3.935877	3.85274	3.734601	4.166933	3.759528	3.994755	4.145833	…	3.222222	3.428571	3.75	2.9375	3.318182	2.659091	2.25	3.375	3.526316	3.184211	3.166667	2.666667	2.8125	2.545455	2.583333	3.416667	3.0	2.75	4.5	3.267857	3.0	2.909091	2.857143	3.678571	3.178571	3.166667	2.666667	3.291667	3.092593	3.263158	3.5	3.0	3.369565	3.263158	3.571429	2.5	2.744898
5	3.951469	4.143125	4.234203	4.07098	4.216733	3.921371	4.013123	3.857326	3.767459	3.938596	4.280078	4.114489	4.250319	3.878066	4.208995	3.899852	3.786963	4.198738	3.788462	3.605442	3.95572	4.137281	3.882759	3.692946	3.203366	3.728785	3.778231	3.572751	3.953148	3.540802	3.898385	3.472033	3.676	3.66404	3.668971	3.880435	…	3.583333	3.244186	3.869565	1.703704	3.416667	2.521739	3.214286	3.384615	3.038462	3.25	2.386364	3.9375	2.365385	3.057692	3.227273	3.40625	3.220588	3.3	3.657895	2.431034	3.216216	3.147059	2.928571	3.989362	3.128205	3.9	3.526316	3.26	2.653846	3.026316	3.107143	3.038462	3.277778	3.625	2.880952	2.442308	2.264706
7	3.730354	3.839698	4.019305	3.788321	4.000476	3.796748	3.809051	3.68019	3.676234	3.755543	4.339982	3.863181	4.072336	3.782511	3.9449	3.879375	3.757656	4.502269	3.525937	3.375661	4.063901	3.865764	3.884908	3.498733	2.864106	3.671037	3.326014	3.081487	4.019151	3.183942	3.727477	3.28777	3.484564	3.597143	3.648005	3.778055	…	3.65	3.348837	4.053846	1.833333	3.181818	2.5	3.323529	3.8125	2.166667	2.384615	1.666667	4.132653	2.75	3.171053	3.173077	3.321429	3.546875	3.45122	3.893939	3.0	3.724638	3.166667	3.25	4.0	2.888889	4.03125	3.833333	3.316667	3.0	2.583333	3.764706	2.708333	3.5	3.5	3.277778	2.928571	1.0
2	4.319876	4.378527	4.416391	4.315249	4.309292	4.187888	4.158518	4.190753	4.208621	4.14554	4.119231	4.017411	4.078606	4.060111	4.058716	3.972491	3.837855	4.113806	4.127123	4.08979	4.140316	3.96871	3.977286	4.181915	3.759015	3.951284	3.994916	3.952915	4.07766	3.861793	3.755793	3.621359	4.045024	3.696608	4.051259	4.140591	…	3.472222	3.6	4.142857	2.5	3.560606	2.770833	3.46875	3.642857	3.242857	3.1	2.797297	3.166667	2.875	3.227273	3.0	3.615385	2.7	3.111111	4.25	2.824324	3.125	3.111111	2.833333	3.833333	3.416667	3.0	3.428571	3.176471	3.342105	3.0	3.325	3.52	3.55	3.464286	3.25	2.543478	2.77027
3	3.845888	4.064403	4.235414	3.985607	4.117811	3.872789	3.90515	3.852941	3.840451	3.872852	4.17204	4.010347	4.163093	3.859091	4.113242	3.813718	3.767078	4.079365	3.774575	3.628505	3.896067	4.024083	3.840262	3.64818	3.083072	3.719661	3.807364	3.51514	3.952791	3.487327	3.806662	3.52854	3.660504	3.652157	3.708262	3.883149	…	3.552632	3.225	3.571429	2.217391	3.0	3.36	3.642857	3.75	3.368421	2.958333	2.75	3.441176	2.566667	3.392857	3.144737	3.295455	3.409091	4.017857	3.583333	1.722222	3.178571	3.285714	2.897436	3.833333	3.242857	3.846154	3.75	3.352941	3.0	2.9	2.764706	3.285714	2.5	3.40625	3.25	2.975	1.958333
6	4.295302	4.272826	4.412418	4.237551	4.271272	4.096102	4.082311	4.142607	4.222386	4.061322	4.165673	3.924207	4.071086	4.03984	4.010932	3.972029	3.866807	4.158256	4.118884	4.058869	4.088632	3.909091	4.034478	4.139023	3.744676	3.975093	3.950496	3.897311	4.094394	3.817641	3.715494	3.584189	3.969184	3.673503	4.053136	4.190691	…	3.714286	3.666667	3.208333	2.552632	3.629032	2.666667	3.65625	3.666667	3.575	3.578125	2.777778	4.214286	3.225	3.5	3.375	3.363636	3.4	2.833333	3.25	3.08	2.6	3.818182	3.038462	3.222222	3.0	2.928571	3.0	3.5	3.5	3.357143	3.434783	3.15	3.869565	3.659091	3.4	3.045455	3.0
4	3.958701	3.830385	4.043483	3.94634	3.972787	3.867537	3.850245	3.849648	4.010293	3.867213	4.038576	3.690628	3.815203	3.895341	3.774429	3.916583	3.894775	3.976823	3.978156	3.798644	3.734758	3.601564	3.920548	3.629735	3.465972	3.793473	3.930318	3.779603	3.87114	3.718121	3.678909	3.619175	3.695165	3.757506	3.83805	4.038248	…	3.907895	3.586207	3.628571	3.193878	4.057143	2.768293	3.837662	3.848214	3.611111	3.56383	2.892857	4.009615	2.830645	3.727273	3.271429	3.633333	4.039474	3.695652	4.038462	2.571429	3.453125	3.475	2.617647	3.55	3.464286	4.351351	3.855556	3.433333	3.48913	2.9	3.5	3.554054	3.633333	3.653846	3.244186	2.666667	2.608696
0	4.340546	4.20889	4.516312	4.292983	4.420844	4.117939	4.094925	4.308089	4.296029	4.086756	4.354976	3.790104	4.187628	4.162293	3.865264	4.087312	3.989047	4.312539	4.038957	4.12249	4.295582	3.725758	4.220837	4.10744	3.58329	4.094983	3.775394	3.602766	4.229718	3.613816	3.934152	3.613015	3.984757	3.787655	4.215424	4.273211	…	3.592593	4.0	4.5	1.625	3.897436	3.666667	3.690476	4.2	4.083333	3.208333	2.666667	4.0	2.75	3.1	3.727273	3.722222	3.0	3.142857	4.0	3.375	3.0	3.875	3.166667	3.636364	3.714286	4.295455	4.0	3.96875	3.225	4.0	4.0	3.25	3.857143	3.666667	3.0	1.25	2.5
8	3.95517	4.217858	4.345401	4.075818	4.135966	4.140683	4.180917	4.055403	3.908564	4.142692	4.289042	4.368097	4.132487	3.86259	4.417532	4.016213	3.933724	4.31702	3.993248	3.79918	4.063517	4.283459	3.927787	3.926336	3.319732	3.789293	3.972625	3.771765	4.083798	3.657728	3.828834	3.63146	3.875	3.792329	3.741379	4.134059	…	3.5	3.333333	3.888889	2.4	3.8	2.5	2.8	0.5	4.333333	1.8	2.5	3.833333	3.0	3.458333	3.857143	4.1	3.96875	3.1875	4.5	2.7	3.25	3.5	3.833333	4.0	3.5	4.363636	3.78125	4.5	3.5	3.7	3.8	3.833333	2.0	3.833333	3.5	3.375	3.0

movieId	1	9	5	7	2	3	6	4	0	8
str	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64
"79132"	4.0	4.374224	3.951469	3.730354	4.319876	3.845888	4.295302	3.958701	4.340546	3.95517
"2571"	4.40856	4.48367	4.143125	3.839698	4.378527	4.064403	4.272826	3.830385	4.20889	4.217858
"318"	4.37247	4.472696	4.234203	4.019305	4.416391	4.235414	4.412418	4.043483	4.516312	4.345401
"58559"	4.2607	4.412461	4.07098	3.788321	4.315249	3.985607	4.237551	3.94634	4.292983	4.075818
"2959"	4.28937	4.30737	4.216733	4.000476	4.309292	4.117811	4.271272	3.972787	4.420844	4.135966
…	…	…	…	…	…	…	…	…	…	…
"67620"	3.603448	3.369565	3.277778	3.5	3.55	2.5	3.869565	3.633333	3.857143	2.0
"115967"	3.297872	3.263158	3.625	3.5	3.464286	3.40625	3.659091	3.653846	3.666667	3.833333
"127134"	3.227273	3.571429	2.880952	3.277778	3.25	3.25	3.4	3.244186	3.0	3.5
"185997"	3.157895	2.5	2.442308	2.928571	2.543478	2.975	3.045455	2.666667	1.25	3.375
"251922"	2.543478	2.744898	2.264706	1.0	2.77027	1.958333	3.0	2.608696	2.5	3.0

imdb_id	title	plot_synopsis	tags	split	synopsis_source	imdbId
str	str	str	str	str	str	i64
"tt0057603"	"I tre volti della paura"	"Note: this synopsis is for the orginal Italian release with the segments in this certain order.Boris Karloff introduces three horror tales of the macabre and the supernatural known as the 'Three Face…	"cult, horror, gothic, murder, atmospheric"	"train"	"imdb"	57603
"tt1733125"	"Dungeons & Dragons: The Book of Vile Darkness"	"Two thousand years ago, Nhagruul the Foul, a sorcerer who reveled in corrupting the innocent and the spread of despair, neared the end of his mortal days and was dismayed. Consumed by hatred for the …	"violence"	"train"	"imdb"	1733125
"tt0033045"	"The Shop Around the Corner"	"Matuschek's, a gift store in Budapest, is the workplace of Alfred Kralik (James Stewart) and the newly hi Ed Klara Novak (Margaret Sullavan). At work they constantly irritate each other, but this dai…	"romantic"	"test"	"imdb"	33045
"tt0113862"	"Mr. Holland's Opus"	"Glenn Holland, not a morning person by anyone's standards, is woken up by his wife Iris early one bright September morning in 1964. Glenn has taken a job as a music teacher at the newly renamed John …	"inspiring, romantic, stupid, feel-good"	"train"	"imdb"	113862
"tt0086250"	"Scarface"	"In May 1980, a Cuban man named Tony Montana (Al Pacino) claims asylum, in Florida, USA, and is in search of the "American Dream" after departing Cuba in the Mariel boatlift of 1980. When questioned b…	"cruelty, murder, dramatic, cult, violence, atmospheric, action, romantic, revenge, sadist"	"val"	"imdb"	86250
…	…	…	…	…	…	…
"tt0219952"	"Lucky Numbers"	"In 1988 Russ Richards (John Travolta), the weatherman for a Harrisburg, Pennsylvania television station, is revered as a local celebrity by his viewers, and fame affords him such perks as a reserved …	"comedy, murder"	"test"	"wikipedia"	219952
"tt1371159"	"Iron Man 2"	"In Russia, the media covers Tony Stark's disclosure of his identity as Iron Man. Ivan Vanko, whose father Anton Vanko has just died, sees this and begins building a miniature arc reactor similar to S…	"good versus evil, violence"	"train"	"wikipedia"	1371159
"tt0063443"	"Play Dirty"	"During the North African Campaign in World War II, Captain Douglas (Caine) is a British Petroleum employee seconded to the Royal Engineers to oversee incoming fuel supplies for the British 8th Army. …	"anti war"	"train"	"wikipedia"	63443
"tt0039464"	"High Wall"	"Steven Kenet catches his unfaithful wife in the apartment of Willard I. Whitcombe, her boss, and apparently strangles her. Believing he killed her, he attempts to commit suicide by driving his car in…	"murder"	"test"	"wikipedia"	39464
"tt0235166"	"Against All Hope"	"Sometime in the 1950s in Chicago a man, Cecil Moe (Michael Madsen) returns home from work with his friend, Joe Cleveland. When Cecil arrives home, he finds his wife Jean and her friend Shannon pourin…	"christian film"	"test"	"wikipedia"	235166

movieId	title	genres	imdb_id	plot_synopsis	0	1	2	3	4	5	6	7	8	9
str	str	str	str	str	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64
"79132"	"Inception (2010)"	"Action\|Crime\|Drama\|Mystery\|Sci-Fi\|Thriller\|IMAX"	"tt1375666"	"A young man, exhausted and delirious, washes up on a beach, looking up momentarily to see two young children (Claire Geare and Magnus Nolan) playing in the sand before he passes out. An armed guard (…	4.340546	4.0	4.319876	3.845888	3.958701	3.951469	4.295302	3.730354	3.95517	4.374224
"2571"	"Matrix, The (1999)"	"Action\|Sci-Fi\|Thriller"	"tt0133093"	"The screen is filled with green, cascading code which gives way to the title, The Matrix.A phone rings and text appears on the screen: "Call trans opt: received. 2-19-98 13:24:18 REC: Log>" As a conv…	4.20889	4.40856	4.378527	4.064403	3.830385	4.143125	4.272826	3.839698	4.217858	4.48367
"318"	"Shawshank Redemption, The (1994)"	"Crime\|Drama"	"tt0111161"	"In 1947, Andy Dufresne (Tim Robbins), a banker in Maine, is convicted of murdering his wife and her lover, a golf pro. Since the state of Maine has no death penalty, he is given two consecutive life …	4.516312	4.37247	4.416391	4.235414	4.043483	4.234203	4.412418	4.019305	4.345401	4.472696
"58559"	"Dark Knight, The (2008)"	"Action\|Crime\|Drama\|IMAX"	"tt0468569"	"The movie begins with a gang of men with clown masks breaking into the bank where the mob has a large portion of their money stashed. It begins with five clowns, each getting a cut of the spoils. The…	4.292983	4.2607	4.315249	3.985607	3.94634	4.07098	4.237551	3.788321	4.075818	4.412461
"2959"	"Fight Club (1999)"	"Action\|Crime\|Drama\|Thriller"	"tt0137523"	"We back out of the webbing of neurons and brain cells as the title credits appear, finding ourselves emerging from the sweat-glistened skin of the protagonist: our narrator (Edward Norton), as he loo…	4.420844	4.28937	4.309292	4.117811	3.972787	4.216733	4.271272	4.000476	4.135966	4.30737
…	…	…	…	…	…	…	…	…	…	…	…	…	…	…
"93819"	"Absentia (2011)"	"Horror"	"tt1610996"	"As the film begins we see Tricia removing old posters and replacing them with new ones. When she comes home, she finds that her younger sister Callie has arrived. She comes to live with her as the pr…	3.875	3.272727	3.111111	3.285714	3.475	3.147059	3.818182	3.166667	3.5	2.909091
"81"	"Things to Do in Denver When You're Dead (1995)"	"Crime\|Drama\|Romance"	"tt0114660"	"Jimmy "The Saint" Tosnia is an ex-gangster living in Denver. Jimmy left the criminal world, to "go straight" with his "Afterlife Advice" business, where dying people videotape messages for their love…	4.0	2.945652	3.0	2.9	2.9	3.026316	3.357143	2.583333	3.7	3.263158
"3952"	"Contender, The (2000)"	"Drama\|Thriller"	"tt0208874"	"Second-term Democratic U.S. President Jackson Evans must select a new Vice President following the sudden death of his previous vice president. The obvious choice seems to be Virginia Governor Jack H…	4.0	3.518868	3.325	2.764706	3.5	3.107143	3.434783	3.764706	3.8	3.5
"7976"	"Ken Park (2002)"	"Drama"	"tt0209077"	"The opening of the film depicts teenager Ken Park (nicknamed "Krap Nek", which is his first and last name spelled and pronounced backward) skateboarding across Visalia, California. He arrives at a sk…	3.25	2.913043	3.52	3.285714	3.554054	3.038462	3.15	2.708333	3.833333	3.0
"67620"	"Nothing But the Truth (2008)"	"Drama\|Thriller"	"tt1073241"	"Robert Bennett (Richard Dix) is a stockbroker who is very carefree with other people's money. Encouraging clients to buy stocks in companies that are failing is all in a day's work to him. His fiancé…	3.857143	3.603448	3.55	2.5	3.633333	3.277778	3.869565	3.5	2.0	3.369565

A
f64
0.416667